Initial analysis on song lyrics, using #tidytex

Libraries

library(readr)
library(tidyverse)

── Attaching packages ───────────────────────────────────────── tidyverse 1.2.1 ──
✔ ggplot2 2.2.1     ✔ purrr   0.2.4
✔ tibble  1.4.2     ✔ dplyr   0.7.4
✔ tidyr   0.7.2     ✔ stringr 1.3.0
✔ ggplot2 2.2.1     ✔ forcats 0.3.0
── Conflicts ──────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()

library(stringr)
library(tidytext)
library(lubridate)

Attaching package: ‘lubridate’

The following object is masked from ‘package:base’:

    date

library(wordcloud)

Loading required package: RColorBrewer

library(topicmodels)
library(tm)

Loading required package: NLP

Attaching package: ‘NLP’

The following object is masked from ‘package:ggplot2’:

    annotate

library(stopwords)

Attaching package: ‘stopwords’

The following object is masked from ‘package:tm’:

    stopwords

library(quanteda)

package ‘quanteda’ was built under R version 3.4.4Package version: 1.1.1
Parallel computing: 2 of 4 threads used.
See https://quanteda.io for tutorials and examples.

Attaching package: ‘quanteda’

The following objects are masked from ‘package:tm’:

    as.DocumentTermMatrix, stopwords

The following object is masked from ‘package:utils’:

    View

Insert data

songs <- read_csv("./data/songs.csv")

Parsed with column specification:
cols(
  index = col_integer(),
  song = col_character(),
  year = col_integer(),
  artist = col_character(),
  genre = col_character(),
  lyrics = col_character()
)

|=======                                                           |  10%   33 MB
|=======                                                           |  11%   34 MB
|=======                                                           |  11%   36 MB
|========                                                          |  12%   37 MB
|========                                                          |  12%   39 MB
|========                                                          |  13%   40 MB
|========                                                          |  13%   41 MB
|=========                                                         |  13%   42 MB
|=========                                                         |  14%   44 MB
|=========                                                         |  14%   45 MB
|==========                                                        |  15%   47 MB
|==========                                                        |  15%   49 MB
|==========                                                        |  16%   50 MB
|===========                                                       |  16%   51 MB
|===========                                                       |  17%   53 MB
|===========                                                       |  17%   54 MB
|============                                                      |  18%   55 MB
|============                                                      |  18%   56 MB
|============                                                      |  18%   58 MB
|============                                                      |  19%   59 MB
|=============                                                     |  19%   61 MB
|=============                                                     |  20%   62 MB
|=============                                                     |  20%   64 MB
|==============                                                    |  21%   65 MB
|==============                                                    |  21%   66 MB
|==============                                                    |  22%   68 MB
|===============                                                   |  22%   69 MB
|===============                                                   |  23%   71 MB
|===============                                                   |  23%   72 MB
|================                                                  |  23%   74 MB
|================                                                  |  24%   75 MB
|================                                                  |  24%   76 MB
|================                                                  |  25%   78 MB
|=================                                                 |  25%   79 MB
|=================                                                 |  26%   81 MB
|=================                                                 |  26%   82 MB
|==================                                                |  26%   83 MB
|==================                                                |  27%   84 MB
|==================                                                |  27%   86 MB
|==================                                                |  28%   87 MB
|===================                                               |  28%   89 MB
|===================                                               |  29%   90 MB
|===================                                               |  29%   92 MB
|====================                                              |  30%   93 MB
|====================                                              |  30%   94 MB
|====================                                              |  31%   96 MB
|=====================                                             |  31%   97 MB
|=====================                                             |  31%   99 MB
|=====================                                             |  32%  100 MB
|======================                                            |  33%  102 MB
|======================                                            |  33%  104 MB
|======================                                            |  34%  105 MB
|=======================                                           |  34%  107 MB
|=======================                                           |  35%  110 MB
|========================                                          |  36%  111 MB
|========================                                          |  36%  113 MB
|========================                                          |  37%  115 MB
|=========================                                         |  37%  116 MB
|=========================                                         |  38%  118 MB
|=========================                                         |  38%  119 MB
|==========================                                        |  39%  120 MB
|==========================                                        |  39%  122 MB
|==========================                                        |  39%  123 MB
|==========================                                        |  40%  124 MB
|===========================                                       |  40%  126 MB
|===========================                                       |  41%  127 MB
|===========================                                       |  41%  128 MB
|============================                                      |  42%  130 MB
|============================                                      |  42%  131 MB
|============================                                      |  43%  133 MB
|=============================                                     |  43%  134 MB
|=============================                                     |  43%  136 MB
|=============================                                     |  44%  137 MB
|==============================                                    |  44%  139 MB
|==============================                                    |  45%  140 MB
|==============================                                    |  45%  141 MB
|===============================                                   |  46%  143 MB
|===============================                                   |  46%  144 MB
|===============================                                   |  47%  145 MB
|===============================                                   |  47%  147 MB
|================================                                  |  48%  149 MB
|================================                                  |  48%  150 MB
|================================                                  |  49%  152 MB
|=================================                                 |  49%  153 MB
|=================================                                 |  49%  154 MB
|=================================                                 |  50%  156 MB
|==================================                                |  50%  157 MB
|==================================                                |  51%  158 MB
|==================================                                |  51%  160 MB
|==================================                                |  52%  161 MB
|===================================                               |  52%  162 MB
|===================================                               |  53%  164 MB
|===================================                               |  53%  166 MB
|====================================                              |  54%  167 MB
|====================================                              |  54%  168 MB
|====================================                              |  54%  169 MB
|=====================================                             |  55%  171 MB
|=====================================                             |  55%  172 MB
|=====================================                             |  56%  174 MB
|======================================                            |  56%  175 MB
|======================================                            |  57%  177 MB
|======================================                            |  57%  178 MB
|======================================                            |  58%  179 MB
|=======================================                           |  58%  181 MB
|=======================================                           |  58%  182 MB
|=======================================                           |  59%  184 MB
|========================================                          |  59%  185 MB
|========================================                          |  60%  186 MB
|========================================                          |  60%  187 MB
|========================================                          |  61%  189 MB
|=========================================                         |  61%  190 MB
|=========================================                         |  62%  192 MB
|=========================================                         |  62%  193 MB
|==========================================                        |  62%  194 MB
|==========================================                        |  63%  196 MB
|==========================================                        |  63%  197 MB
|===========================================                       |  64%  198 MB
|===========================================                       |  64%  200 MB
|===========================================                       |  65%  201 MB
|===========================================                       |  65%  202 MB
|============================================                      |  65%  204 MB
|============================================                      |  66%  205 MB
|============================================                      |  66%  207 MB
|=============================================                     |  67%  208 MB
|=============================================                     |  67%  210 MB
|=============================================                     |  68%  211 MB
|==============================================                    |  69%  213 MB
|==============================================                    |  69%  215 MB
|==============================================                    |  69%  216 MB
|===============================================                   |  70%  217 MB
|===============================================                   |  70%  219 MB
|===============================================                   |  71%  221 MB
|================================================                  |  72%  222 MB
|================================================                  |  72%  224 MB
|================================================                  |  72%  225 MB
|=================================================                 |  73%  226 MB
|=================================================                 |  73%  228 MB
|=================================================                 |  74%  229 MB
|==================================================                |  74%  231 MB
|==================================================                |  75%  232 MB
|==================================================                |  75%  233 MB
|==================================================                |  75%  235 MB
|===================================================               |  76%  236 MB
|===================================================               |  76%  237 MB
|===================================================               |  77%  239 MB
|====================================================              |  77%  240 MB
|====================================================              |  78%  242 MB
|====================================================              |  78%  243 MB
|====================================================              |  78%  244 MB
|=====================================================             |  79%  245 MB
|=====================================================             |  79%  247 MB
|=====================================================             |  80%  248 MB
|======================================================            |  80%  250 MB
|======================================================            |  81%  251 MB
|======================================================            |  81%  253 MB
|=======================================================           |  82%  254 MB
|=======================================================           |  82%  255 MB
|=======================================================           |  82%  256 MB
|=======================================================           |  83%  258 MB
|========================================================          |  83%  259 MB
|========================================================          |  84%  261 MB
|========================================================          |  84%  262 MB
|=========================================================         |  85%  264 MB
|=========================================================         |  85%  265 MB
|==========================================================        |  86%  268 MB
|==========================================================        |  87%  270 MB
|==========================================================        |  87%  271 MB
|===========================================================       |  88%  273 MB
|===========================================================       |  88%  274 MB
|===========================================================       |  89%  275 MB
|===========================================================       |  89%  277 MB
|============================================================      |  89%  278 MB
|============================================================      |  90%  279 MB
|============================================================      |  90%  280 MB
|============================================================      |  91%  281 MB
|=============================================================     |  91%  283 MB
|=============================================================     |  91%  284 MB
|=============================================================     |  92%  285 MB
|==============================================================    |  92%  287 MB
|==============================================================    |  93%  288 MB
|==============================================================    |  93%  289 MB
|==============================================================    |  94%  291 MB
|===============================================================   |  94%  292 MB
|===============================================================   |  95%  294 MB
|================================================================  |  95%  296 MB
|================================================================  |  96%  297 MB
|================================================================  |  96%  299 MB
|================================================================= |  97%  300 MB
|================================================================= |  97%  302 MB
|================================================================= |  98%  303 MB
|==================================================================|  98%  305 MB
|==================================================================|  98%  306 MB
|==================================================================|  99%  307 MB
|==================================================================|  99%  309 MB
|===================================================================| 100%  309 MB

DATA CLEANSING

Filter data, handle missing values

# Keep songs with 10 or more characters in the lyrics
songs <- 
  songs %>% 
  filter(str_length(lyrics) > 10)
  
# Detect the language of the song
library(cld3)
songs$lang <- detect_language(songs$lyrics)
# Filter the songs that are certainly english, the genre is not missing or 
# not available and the year is 1970 or later
songs <- 
  songs %>%
  filter(lang == "en" & 
           is.na(genre) == FALSE &
           year >= 1970 &
           genre != "Not Available")
  
songs$characters <- str_count(songs$lyrics)

_text cleaning

# Create a vector with stopwords
stopwords <- c(stopwords())
  
  
# Perform cleansing & stemming in lsong lyrics text
songs$lyrics <- tolower(songs$lyrics)
songs$lyrics <- removePunctuation(songs$lyrics)
songs$lyrics <- removeNumbers(songs$lyrics)
songs$lyrics <- stripWhitespace(songs$lyrics)
songs$lyrics <- removeWords(songs$lyrics, stopwords)
songs$lyrics <- stemDocument(songs$lyrics)
saveRDS(songs, file = "data/cleaned_data.RDS")

EXPLORATORY ANALYSIS

Are there any differences between different music genres

_How many songs per artist

songs <- readRDS(file = "data/cleaned_data.RDS")
songs %>% 
  group_by(artist) %>% 
  count() %>% 
  filter(n > 20) %>% 
  arrange(-n)

• Not really informative. maybe

_How many songs per genre

songs %>% 
  group_by(genre) %>% 
  count() %>% 
  arrange(-n) %>% 
  ggplot() +
  geom_col(aes(reorder(genre, -n), n))

• could choose specific genres for comparison. e.g. Rock, pop, hip-hop, metal, country, jazz, electronic.

_What kind of music uses more words

## number of characters per song
songs %>% 
  ggplot() +
  geom_boxplot(aes(genre, characters))

• Hip-Hop seems to be the significantly different music genre.

_How many songs per genres & Artists

songs %>% 
  group_by(genre, artist) %>% 
  count() %>% 
  arrange(genre, -n)

• Not informative

_How many songs per genres & Artists

songs %>% 
  group_by(artist, genre) %>% 
  count() %>% 
  arrange(artist, genre, -n) %>% 
  group_by(artist) %>% 
  summarise(N = n(), total_songs = sum(n)) %>% 
  arrange(-N)

** Possible problem as every artist belongs to just one genres. But it is known that a lot of artists do change their music genre through their career.

_How many songs per Year

songs %>% 
  group_by(year) %>% 
  count() 

*** There are a few songs with wrong year … ++

_How many songs per Decade and Genre

songs %>% 
  mutate(date = as_date(paste(as.character(songs$year), "-01", "-01"))) %>% 
  mutate(decade = floor_date(date, years(10))) %>% 
  group_by(decade, genre) %>% 
  summarise(N = n()) %>% 
  mutate(freq = round(N/sum(N), 2)) %>% 
  select(-N) %>% 
  spread(key = decade, value = freq) 

songs %>% 
  mutate(date = as_date(paste(as.character(songs$year), "-01", "-01"))) %>% 
  mutate(decade = floor_date(date, years(5))) %>% 
  group_by(decade, genre) %>% 
  summarise(N = n()) %>% 
  mutate(freq = round(N/sum(N), 2)) %>% 
  filter(genre %in% c("Country", "Hip-Hop", "Metal", "Pop", "Rock")) %>% 
  ggplot(aes(decade, freq, colour = genre)) +
  # geom_line() +
  geom_smooth(se = FALSE) +
  scale_y_continuous(labels = scales::percent_format()) +
  ggtitle("Smoothed frequency of total songs per Music Genre (top 5)")

NA
NA
NA

** I think it would be wrong to say things about music genres. Rock is Falling - Hip hop is raising ** could make a statistical test for counts, chi-square to show differences

_How many characters per Decade & Decade+Genre

songs %>% 
  mutate(date = as_date(paste(as.character(songs$year), "-01", "-01"))) %>% 
  mutate(decade = floor_date(date, years(10))) %>% 
  group_by(decade) %>% 
  summarise(characters = mean(characters, na.rm = TRUE)) %>% 
  # mutate(freq = round(N/sum(N), 2)) %>% 
  # select(-N) %>% 
  spread(key = decade, value = characters)

songs %>% 
  mutate(date = as_date(paste(as.character(songs$year), "-01", "-01"))) %>% 
  mutate(decade = floor_date(date, years(10))) %>% 
  group_by(decade, genre) %>% 
  summarise(characters = round(mean(characters, na.rm = TRUE), 0)) %>% 
  # mutate(freq = round(N/sum(N), 2)) %>% 
  # select(-N) %>% 
  spread(key = decade, value = characters) 

songs %>% 
  mutate(date = as_date(paste(as.character(songs$year), "-01", "-01"))) %>% 
  mutate(decade = floor_date(date, years(10))) %>% 
  group_by(decade, genre) %>% 
  summarise(characters = round(mean(characters, na.rm = TRUE), 0)) %>% 
 ggplot(aes(decade, characters, colour = genre)) +
  geom_line() +
  # geom_smooth(se = FALSE) +
  ggtitle("How much lyrics per song for Music Genre")

  
songs %>% 
  mutate(date = as_date(paste(as.character(songs$year), "-01", "-01"))) %>% 
  mutate(decade = floor_date(date, years(10))) %>% 
  group_by(decade) %>% 
  summarise(characters = round(mean(characters, na.rm = TRUE), 0)) %>% 
 ggplot(aes(decade, characters)) +
  geom_line() +
  # geom_smooth(se = FALSE) +
  ggtitle("How much lyrics per song per 5 year interval")

songs %>% 
  group_by(genre) %>% 
  summarise(characters = round(mean(characters, na.rm = TRUE), 0)) %>% 
ggplot(aes(reorder(genre, -characters), characters)) +
  geom_col() +
  ggtitle("Average lyrics characters per song for each music genre")

*** It seems that Hip-hop songs contain much more lyrics (almost double). ????

_Wordcloud

words <- 
  songs %>%
  unnest_tokens(word, lyrics) %>% 
  group_by(word) %>% 
  count() %>% 
  arrange(-n) %>% 
  head(100)
wordcloud(words = words$word, freq = words$n, max.words = 100)

_Wordcloud per genre - chech older … comparing

words <- 
  songs %>%
  unnest_tokens(word, lyrics) %>% 
  group_by(genre, word) %>% 
  count() %>% 
  arrange(-n) %>% 
  group_by(genre) %>% 
  top_n(n = 100, wt = n)
genres <- unique(words$genre)
for(i in 1:length(genres)){
  temp <- filter(words, genre == genres[i])
  
  # Create a word cloud
  par(bg="grey30")
  #png(file="/home/manos4/Analysis/002_Content/output/figs/WordCloud_all_posts.png",width=1000,height=700, bg="grey30")
  wordcloud(words = temp$word, freq = temp$n, col=terrain.colors(length(temp$word), alpha=0.9), random.order=FALSE, rot.per=0.3 )
  title(main =  genres[i] , font.main = 1, col.main = "cornsilk3", cex.main = 1.2)
  #dev.off()
}

pain could not be fit on page. It will not be plotted.light could not be fit on page. It will not be plotted.god could not be fit on page. It will not be plotted.ive could not be fit on page. It will not be plotted.hand could not be fit on page. It will not be plotted.insid could not be fit on page. It will not be plotted.say could not be fit on page. It will not be plotted.face could not be fit on page. It will not be plotted.got could not be fit on page. It will not be plotted.need could not be fit on page. It will not be plotted.kill could not be fit on page. It will not be plotted.caus could not be fit on page. It will not be plotted.fire could not be fit on page. It will not be plotted.sky could not be fit on page. It will not be plotted.right could not be fit on page. It will not be plotted.there could not be fit on page. It will not be plotted.wont could not be fit on page. It will not be plotted.lost could not be fit on page. It will not be plotted.hell could not be fit on page. It will not be plotted.black could not be fit on page. It will not be plotted.stand could not be fit on page. It will not be plotted.leav could not be fit on page. It will not be plotted.thing could not be fit on page. It will not be plotted.last could not be fit on page. It will not be plotted.hear could not be fit on page. It will not be plotted.word could not be fit on page. It will not be plotted.hate could not be fit on page. It will not be plotted.well could not be fit on page. It will not be plotted.everi could not be fit on page. It will not be plotted.tear could not be fit on page. It will not be plotted.keep could not be fit on page. It will not be plotted.head could not be fit on page. It will not be plotted.think could not be fit on page. It will not be plotted.wait could not be fit on page. It will not be plotted.chorus could not be fit on page. It will not be plotted.forev could not be fit on page. It will not be plotted.alon could not be fit on page. It will not be plotted.

gone could not be fit on page. It will not be plotted.thought could not be fit on page. It will not be plotted.

gone could not be fit on page. It will not be plotted.

NA

*** Print top-5 genres

_How much they repeat themselves - Rechedule code from ….

MODELLING ON GENRES

_Create the model

# _Build Model ###############################################################
  
# split into words
by_word <- 
  songs %>% 
  unnest_tokens(word, lyrics)
# find document-word counts
word_counts <- 
  by_word %>%
  count(song, word, sort = TRUE) %>%
  ungroup()
word_counts

songs_dtm <- word_counts %>%
  cast_dtm(song, word, n)
songs_dtm

<<DocumentTermMatrix (documents: 151622, terms: 181138)>>
Non-/sparse entries: 13271052/27451234784
Sparsity           : 100%
Maximal term length: 247
Weighting          : term frequency (tf)

#songs_lda <- LDA(songs_dtm, k = 4, control = list(seed = 1234))
songs_lda <- LDA(songs_dtm, k = 3, control = list(seed = 1234))
#songs_lda_2 <- LDA(songs_dtm, k = 2, control = list(seed = 1234))
save(songs_lda, file = "objects/songs_lda_3.RDA")

• Create another model with 2 groups to check hip-hop & metal Vs the rest

_Print probabilities for each song to belong to each topic

load(file = "objects/songs_lda_3.RDA")
songs_gamma <- 
  tidy(songs_lda, matrix = "gamma") %>% 
  arrange(document, topic)

_Top terms per Topic

 # _Calculate Tables ##########################################################
  library(tidytext)
  
  ap_topics <- tidy(songs_lda, matrix = "gamma")
  
  top_terms <- 
    ap_topics %>%
    group_by(topic) %>%
    top_n(10, gamma) %>%
    ungroup() %>%
    arrange(topic, -gamma) %>% 
    rename(prob = gamma)
  
  top_terms

_Are topics and genres related?

# _Calculate Tables ##########################################################
library(tidytext)
  
ap_topics <- tidy(songs_lda, matrix = "gamma")
  
test <- right_join(ap_topics, songs[, c("song", "genre")], by = c("document" = "song"))
 
# reorder titles in order of topic 1, topic 2, etc before plotting
test %>%
  mutate(genre = reorder(genre, gamma * topic)) %>%
  ggplot(aes(factor(topic), gamma)) +
  geom_boxplot() +
  facet_wrap(~ genre)

*** 1 Topic are most of genres. POP, R&B, Country… *** The 2nd topic is dominated by Hip-Hop. *** The 3rd topic is dominated by Metal genre. although there is a significant contribution of Folk music